SETUP

Output dir

path <- paste("results/")
ifelse(!dir.exists(path), dir.create(path), FALSE)
[1] FALSE
savedir <- paste(path,"Probing",sep='')
ifelse(!dir.exists(savedir), dir.create(savedir), FALSE)
[1] FALSE

READ DATA

model2plausibility_dir = "../probing/results/model2plausibility"
datafile = "all_combined_kfold_new.csv"

dat = read.csv(paste(model2plausibility_dir, datafile, sep="/"))


dat <- subset(dat, select = -X)

dat$Layer = as.factor(dat$Layer)
dat$Model = factor(dat$Model, levels=c("bert-large-cased", "roberta-large", "gpt2-xl"))

STATS

dat.DTFit = dat %>% filter(Dataset=="DTFit")

m.DTFit = lm(Accuracy~Layer:Model, data=dat.DTFit)
summary(m.DTFit)

Call:
lm(formula = Accuracy ~ Layer:Model, data = dat.DTFit)

Residuals:
      Min        1Q    Median        3Q       Max 
-0.138173 -0.027300  0.000385  0.027628  0.125481 

Coefficients: (49 not defined because of singularities)
                                Estimate Std. Error t value Pr(>|t|)    
(Intercept)                    0.7958654  0.0132318  60.148  < 2e-16 ***
Layer1:Modelbert-large-cased  -0.2958654  0.0187126 -15.811  < 2e-16 ***
Layer2:Modelbert-large-cased  -0.1811859  0.0187126  -9.683  < 2e-16 ***
Layer3:Modelbert-large-cased  -0.1535577  0.0187126  -8.206 8.26e-16 ***
Layer4:Modelbert-large-cased  -0.1322115  0.0187126  -7.065 3.30e-12 ***
Layer5:Modelbert-large-cased  -0.1462180  0.0187126  -7.814 1.61e-14 ***
Layer6:Modelbert-large-cased  -0.1245513  0.0187126  -6.656 4.99e-11 ***
Layer7:Modelbert-large-cased  -0.1069872  0.0187126  -5.717 1.49e-08 ***
Layer8:Modelbert-large-cased  -0.0882372  0.0187126  -4.715 2.81e-06 ***
Layer9:Modelbert-large-cased  -0.0250641  0.0187126  -1.339 0.180785    
Layer10:Modelbert-large-cased -0.0201282  0.0187126  -1.076 0.282384    
Layer11:Modelbert-large-cased -0.0289744  0.0187126  -1.548 0.121895    
Layer12:Modelbert-large-cased  0.0140064  0.0187126   0.749 0.454361    
Layer13:Modelbert-large-cased  0.0029487  0.0187126   0.158 0.874825    
Layer14:Modelbert-large-cased -0.0009936  0.0187126  -0.053 0.957667    
Layer15:Modelbert-large-cased  0.0115064  0.0187126   0.615 0.538781    
Layer16:Modelbert-large-cased  0.0355128  0.0187126   1.898 0.058055 .  
Layer17:Modelbert-large-cased  0.0389423  0.0187126   2.081 0.037721 *  
Layer18:Modelbert-large-cased  0.0417308  0.0187126   2.230 0.025998 *  
Layer19:Modelbert-large-cased  0.0213141  0.0187126   1.139 0.255008    
Layer20:Modelbert-large-cased  0.0251282  0.0187126   1.343 0.179673    
Layer21:Modelbert-large-cased  0.0190705  0.0187126   1.019 0.308428    
Layer22:Modelbert-large-cased -0.0012179  0.0187126  -0.065 0.948120    
Layer23:Modelbert-large-cased -0.0507051  0.0187126  -2.710 0.006868 ** 
Layer24:Modelbert-large-cased -0.0187821  0.0187126  -1.004 0.315798    
Layer25:Modelbert-large-cased         NA         NA      NA       NA    
Layer26:Modelbert-large-cased         NA         NA      NA       NA    
Layer27:Modelbert-large-cased         NA         NA      NA       NA    
Layer28:Modelbert-large-cased         NA         NA      NA       NA    
Layer29:Modelbert-large-cased         NA         NA      NA       NA    
Layer30:Modelbert-large-cased         NA         NA      NA       NA    
Layer31:Modelbert-large-cased         NA         NA      NA       NA    
Layer32:Modelbert-large-cased         NA         NA      NA       NA    
Layer33:Modelbert-large-cased         NA         NA      NA       NA    
Layer34:Modelbert-large-cased         NA         NA      NA       NA    
Layer35:Modelbert-large-cased         NA         NA      NA       NA    
Layer36:Modelbert-large-cased         NA         NA      NA       NA    
Layer37:Modelbert-large-cased         NA         NA      NA       NA    
Layer38:Modelbert-large-cased         NA         NA      NA       NA    
Layer39:Modelbert-large-cased         NA         NA      NA       NA    
Layer40:Modelbert-large-cased         NA         NA      NA       NA    
Layer41:Modelbert-large-cased         NA         NA      NA       NA    
Layer42:Modelbert-large-cased         NA         NA      NA       NA    
Layer43:Modelbert-large-cased         NA         NA      NA       NA    
Layer44:Modelbert-large-cased         NA         NA      NA       NA    
Layer45:Modelbert-large-cased         NA         NA      NA       NA    
Layer46:Modelbert-large-cased         NA         NA      NA       NA    
Layer47:Modelbert-large-cased         NA         NA      NA       NA    
Layer48:Modelbert-large-cased         NA         NA      NA       NA    
Layer1:Modelroberta-large     -0.2958654  0.0187126 -15.811  < 2e-16 ***
Layer2:Modelroberta-large     -0.2882051  0.0187126 -15.402  < 2e-16 ***
Layer3:Modelroberta-large     -0.1735897  0.0187126  -9.277  < 2e-16 ***
Layer4:Modelroberta-large     -0.1853205  0.0187126  -9.904  < 2e-16 ***
Layer5:Modelroberta-large     -0.1645833  0.0187126  -8.795  < 2e-16 ***
Layer6:Modelroberta-large     -0.1020513  0.0187126  -5.454 6.45e-08 ***
Layer7:Modelroberta-large     -0.1158654  0.0187126  -6.192 9.18e-10 ***
Layer8:Modelroberta-large     -0.1599679  0.0187126  -8.549  < 2e-16 ***
Layer9:Modelroberta-large     -0.0980449  0.0187126  -5.240 2.02e-07 ***
Layer10:Modelroberta-large    -0.0654808  0.0187126  -3.499 0.000490 ***
Layer11:Modelroberta-large    -0.0311539  0.0187126  -1.665 0.096303 .  
Layer12:Modelroberta-large    -0.0148077  0.0187126  -0.791 0.428973    
Layer13:Modelroberta-large     0.0317308  0.0187126   1.696 0.090305 .  
Layer14:Modelroberta-large     0.0301282  0.0187126   1.610 0.107752    
Layer15:Modelroberta-large     0.0532372  0.0187126   2.845 0.004546 ** 
Layer16:Modelroberta-large     0.0468590  0.0187126   2.504 0.012458 *  
Layer17:Modelroberta-large     0.0256090  0.0187126   1.369 0.171498    
Layer18:Modelroberta-large     0.0681731  0.0187126   3.643 0.000285 ***
Layer19:Modelroberta-large     0.0604808  0.0187126   3.232 0.001275 ** 
Layer20:Modelroberta-large     0.0466667  0.0187126   2.494 0.012822 *  
Layer21:Modelroberta-large     0.0554808  0.0187126   2.965 0.003111 ** 
Layer22:Modelroberta-large     0.0367308  0.0187126   1.963 0.049979 *  
Layer23:Modelroberta-large     0.0554808  0.0187126   2.965 0.003111 ** 
Layer24:Modelroberta-large     0.0467308  0.0187126   2.497 0.012700 *  
Layer25:Modelroberta-large            NA         NA      NA       NA    
Layer26:Modelroberta-large            NA         NA      NA       NA    
Layer27:Modelroberta-large            NA         NA      NA       NA    
Layer28:Modelroberta-large            NA         NA      NA       NA    
Layer29:Modelroberta-large            NA         NA      NA       NA    
Layer30:Modelroberta-large            NA         NA      NA       NA    
Layer31:Modelroberta-large            NA         NA      NA       NA    
Layer32:Modelroberta-large            NA         NA      NA       NA    
Layer33:Modelroberta-large            NA         NA      NA       NA    
Layer34:Modelroberta-large            NA         NA      NA       NA    
Layer35:Modelroberta-large            NA         NA      NA       NA    
Layer36:Modelroberta-large            NA         NA      NA       NA    
Layer37:Modelroberta-large            NA         NA      NA       NA    
Layer38:Modelroberta-large            NA         NA      NA       NA    
Layer39:Modelroberta-large            NA         NA      NA       NA    
Layer40:Modelroberta-large            NA         NA      NA       NA    
Layer41:Modelroberta-large            NA         NA      NA       NA    
Layer42:Modelroberta-large            NA         NA      NA       NA    
Layer43:Modelroberta-large            NA         NA      NA       NA    
Layer44:Modelroberta-large            NA         NA      NA       NA    
Layer45:Modelroberta-large            NA         NA      NA       NA    
Layer46:Modelroberta-large            NA         NA      NA       NA    
Layer47:Modelroberta-large            NA         NA      NA       NA    
Layer48:Modelroberta-large            NA         NA      NA       NA    
Layer1:Modelgpt2-xl           -0.2769551  0.0187126 -14.800  < 2e-16 ***
Layer2:Modelgpt2-xl           -0.1347115  0.0187126  -7.199 1.32e-12 ***
Layer3:Modelgpt2-xl           -0.1043269  0.0187126  -5.575 3.31e-08 ***
Layer4:Modelgpt2-xl           -0.0983974  0.0187126  -5.258 1.83e-07 ***
Layer5:Modelgpt2-xl           -0.1082692  0.0187126  -5.786 1.01e-08 ***
Layer6:Modelgpt2-xl           -0.0917308  0.0187126  -4.902 1.13e-06 ***
Layer7:Modelgpt2-xl           -0.0555449  0.0187126  -2.968 0.003077 ** 
Layer8:Modelgpt2-xl           -0.0580449  0.0187126  -3.102 0.001985 ** 
Layer9:Modelgpt2-xl           -0.0438782  0.0187126  -2.345 0.019260 *  
Layer10:Modelgpt2-xl          -0.0652564  0.0187126  -3.487 0.000513 ***
Layer11:Modelgpt2-xl          -0.0579167  0.0187126  -3.095 0.002031 ** 
Layer12:Modelgpt2-xl          -0.0326923  0.0187126  -1.747 0.080979 .  
Layer13:Modelgpt2-xl          -0.0136859  0.0187126  -0.731 0.464749    
Layer14:Modelgpt2-xl          -0.0201923  0.0187126  -1.079 0.280855    
Layer15:Modelgpt2-xl          -0.0100321  0.0187126  -0.536 0.592019    
Layer16:Modelgpt2-xl          -0.0074039  0.0187126  -0.396 0.692453    
Layer17:Modelgpt2-xl          -0.0238461  0.0187126  -1.274 0.202886    
Layer18:Modelgpt2-xl          -0.0248718  0.0187126  -1.329 0.184150    
Layer19:Modelgpt2-xl          -0.0302564  0.0187126  -1.617 0.106264    
Layer20:Modelgpt2-xl           0.0000641  0.0187126   0.003 0.997268    
Layer21:Modelgpt2-xl           0.0190064  0.0187126   1.016 0.310056    
Layer22:Modelgpt2-xl          -0.0098397  0.0187126  -0.526 0.599137    
Layer23:Modelgpt2-xl           0.0028526  0.0187126   0.152 0.878875    
Layer24:Modelgpt2-xl           0.0164423  0.0187126   0.879 0.379821    
Layer25:Modelgpt2-xl           0.0076923  0.0187126   0.411 0.681118    
Layer26:Modelgpt2-xl          -0.0035577  0.0187126  -0.190 0.849257    
Layer27:Modelgpt2-xl          -0.0261538  0.0187126  -1.398 0.162574    
Layer28:Modelgpt2-xl          -0.0148077  0.0187126  -0.791 0.428973    
Layer29:Modelgpt2-xl          -0.0087500  0.0187126  -0.468 0.640189    
Layer30:Modelgpt2-xl          -0.0313141  0.0187126  -1.673 0.094606 .  
Layer31:Modelgpt2-xl          -0.0162820  0.0187126  -0.870 0.384481    
Layer32:Modelgpt2-xl          -0.0264744  0.0187126  -1.415 0.157491    
Layer33:Modelgpt2-xl          -0.0248718  0.0187126  -1.329 0.184150    
Layer34:Modelgpt2-xl          -0.0274039  0.0187126  -1.464 0.143432    
Layer35:Modelgpt2-xl          -0.0365064  0.0187126  -1.951 0.051392 .  
Layer36:Modelgpt2-xl          -0.0074359  0.0187126  -0.397 0.691190    
Layer37:Modelgpt2-xl          -0.0149680  0.0187126  -0.800 0.423996    
Layer38:Modelgpt2-xl          -0.0303205  0.0187126  -1.620 0.105527    
Layer39:Modelgpt2-xl          -0.0438141  0.0187126  -2.341 0.019437 *  
Layer40:Modelgpt2-xl          -0.0113461  0.0187126  -0.606 0.544449    
Layer41:Modelgpt2-xl          -0.0250000  0.0187126  -1.336 0.181901    
Layer42:Modelgpt2-xl          -0.0298718  0.0187126  -1.596 0.110777    
Layer43:Modelgpt2-xl          -0.0100961  0.0187126  -0.540 0.589655    
Layer44:Modelgpt2-xl          -0.0173718  0.0187126  -0.928 0.353486    
Layer45:Modelgpt2-xl          -0.0363141  0.0187126  -1.941 0.052629 .  
Layer46:Modelgpt2-xl           0.0017308  0.0187126   0.092 0.926328    
Layer47:Modelgpt2-xl          -0.0239103  0.0187126  -1.278 0.201676    
Layer48:Modelgpt2-xl                  NA         NA      NA       NA    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.04184 on 864 degrees of freedom
Multiple R-squared:  0.7949,    Adjusted R-squared:  0.7724 
F-statistic: 35.25 on 95 and 864 DF,  p-value: < 2.2e-16

READ HUMAN CEILING DATA

dat$Layer = as.numeric(as.character(dat$Layer))


model2plausibility_dir = "../probing/results/model2human_ceiling"

read_data <- function(directory, filename) {
  print(paste(filename))
  metadata = str_split(filename, "_")[[1]]
  d = read.delim(paste(model2plausibility_dir, filename, sep='/'), 
                 header=FALSE, sep=',')
  d = d %>%
    mutate(Dataset = metadata[[2]]) %>%
    mutate(VoiceType = metadata[[3]]) %>%
    mutate(TrialType = metadata[[4]]) %>%
    mutate(TrialType = str_replace(TrialType, ".csv", "")) %>%
    rename(Iteration = V1)  %>%
    rename(Accuracy = V2)  %>%
    mutate(Plot = "HumanCeiling")
}

filenames = list.files(path=model2plausibility_dir, pattern='*.csv')
dat.ceiling = do.call(rbind, lapply(filenames, function(x) read_data(model2plausibility_dir, x)))
[1] "bert-large-cased_DTFit_normal_normal.csv"
[1] "bert-large-cased_EventsAdapt_active-active_AAN-AAN.csv"
[1] "bert-large-cased_EventsAdapt_active-active_AAN-AAR.csv"
[1] "bert-large-cased_EventsAdapt_active-active_AAN-AI.csv"
[1] "bert-large-cased_EventsAdapt_active-active_AI-AAN.csv"
[1] "bert-large-cased_EventsAdapt_active-active_AI-AAR.csv"
[1] "bert-large-cased_EventsAdapt_active-active_AI-AI.csv"
[1] "bert-large-cased_EventsAdapt_active-active_normal-AAR.csv"
[1] "bert-large-cased_EventsAdapt_active-active_normal.csv"
[1] "bert-large-cased_EventsAdapt_active-passive_normal.csv"
[1] "bert-large-cased_EventsAdapt_normal_normal.csv"
[1] "bert-large-cased_EventsAdapt_passive-active_normal.csv"
[1] "bert-large-cased_EventsAdapt_passive-passive_normal.csv"
[1] "bert-large-cased_EventsRev_normal_normal.csv"
add_info <- function(dataframe, model_name) {
  if (grepl("gpt2-xl",model_name) == FALSE){
    toadd = 5
  }else{
    toadd = 10
  }
  dataframe = dataframe %>%
    mutate(Layer = max(subset(dat, Model==model_name)$Layer) + toadd) %>%
    mutate(Model = model_name)
}

models = unique(c(dat$Model))
  
dat.ceiling.full = do.call(rbind, lapply(models, function(x) add_info(dat.ceiling, x)))
merged <- rbind(dat, dat.ceiling.full)

PLOT

breaks function x axis

breaks_fun <- function(x) {
  if (max(x) > 40) {
    c(seq(0, 50, 10), max(x) + 10)
  } else {
    c(seq(0, 25, 5), max(x) + 5)
  }
}

All datasets

plot_data = merged %>% filter(TrialType=="normal", VoiceType=="normal",Plot!="HumanCeiling")
plot_data$Layer = as.numeric(as.character(plot_data$Layer))

ceil_data = merged %>% filter(TrialType=="normal", VoiceType=="normal",Plot=="HumanCeiling")
ceil_summary <- ceil_data %>%
  group_by(Dataset) %>%
  summarise(
    MeanAccuracy = mean(Accuracy),
    sd = sd(Accuracy),
    n = n(),
    SE = sd / sqrt(n)
  )

datasets = c(ceil_summary$Dataset)

ceil_data = ceil_data %>% 
  mutate(MeanAccuracy = NA) %>%
  mutate(SE = NA)

for( dt in datasets){
  ceil_data = ceil_data %>%
    mutate(MeanAccuracy = ifelse(ceil_data$Dataset==dt, subset(ceil_summary, Dataset == dt)$MeanAccuracy, ceil_data$MeanAccuracy)) %>%
    mutate(SE = ifelse(ceil_data$Dataset==dt, subset(ceil_summary, Dataset == dt)$SE, ceil_data$SE))
}

ggplot(data = plot_data,
       mapping = aes(x=Layer, y=Accuracy, color=Dataset, group = Dataset))+
  facet_grid(~Model, scales="free_x")+
  geom_hline(yintercept=.5, linetype='dotted')+
  geom_hline(yintercept=1, linetype='dotted')+
  stat_summary(geom='line', fun='mean')+
  stat_summary(geom='errorbar', fun.data='mean_se',
               size = 0.2, width=0.1)+
  geom_point(data = ceil_data, aes(x=Layer, y=MeanAccuracy, color=Dataset), size=0.2)+
  geom_errorbar(data = ceil_data, aes(x=Layer, ymin=MeanAccuracy-SE, ymax=MeanAccuracy+SE, color=Dataset),width = 0.05) +
  geom_text(data = ceil_data, group=ceil_data$Dataset, x =ifelse(ceil_data$Model=="gpt2-xl", ceil_data$Layer - 6, ceil_data$Layer - 3),  y = 0.98,
            size = 2.5,
            label = "ceiling", 
            colour = "#6d6d6d") +
  theme_classic()+
  scale_x_continuous(breaks=breaks_fun)+
  scale_y_continuous(breaks=seq(0.4,1.2,0.1))


savename <- "model2plausibility_all_datasets_ceiled.png"
ggsave(paste(savedir,savename,sep="/"), width=20, height=8, units='cm', device='tiff', dpi=700)

DTFit

dataset = "DTFit"

ggplot(data = dat %>% filter(Dataset==dataset),
       mapping = aes(x=Layer, y=Accuracy, group=dataset))+
  facet_wrap(~Plot, scales="free_x")+
  scale_x_continuous(breaks = breaks_fun, limits = c(0, NA)) + 
  geom_hline(yintercept=.5, linetype='dotted')+
  geom_hline(yintercept=1, linetype='dotted')+
  stat_summary(geom='line', fun='mean')+
  stat_summary(geom='errorbar', fun.data='mean_se',
               color = 'black', size = 0.5, width=0.1)+
  theme_classic()

savename <- "model2plausibility_DTFit.png"
ggsave(paste(savedir,savename,sep="/"), width=16, height=8, units='cm')

EventsRev

dataset = "EventsRev"

ggplot(data = dat %>% filter(Dataset==dataset),
       mapping = aes(x=Layer, y=Accuracy, group=dataset))+
  facet_grid(~Plot, scales="free_x")+
  scale_x_continuous(breaks = breaks_fun, limits = c(0, NA)) + 
  geom_hline(yintercept=.5, linetype='dotted')+
  geom_hline(yintercept=1, linetype='dotted')+
  stat_summary(geom='line', fun='mean')+
  stat_summary(geom='errorbar', fun.data='mean_se',
               color = 'black', size = 0.5, width=0.1)+
  theme_classic()

savename <- "model2plausibility_EventsRev.png"
ggsave(paste(savedir,savename,sep="/"), width=16, height=8, units='cm')

EventsAdapt

Active Passive

dataset = "EventsAdapt"

ggplot(data = dat %>% filter(Dataset==dataset, TrialType=="normal"),
       mapping = aes(x=Layer, y=Accuracy, color=VoiceType, group=VoiceType))+
  facet_grid(~Model, scales="free_x")+ 
  scale_x_continuous(breaks = breaks_fun, limits = c(0, NA)) + 
  geom_hline(yintercept=.5, linetype='dotted')+
  geom_hline(yintercept=1, linetype='dotted')+
  stat_summary(geom='line', fun='mean')+
  stat_summary(geom='errorbar', fun.data='mean_se',
               size = 0.2, width=0)+
  theme_classic()

savename <- "model2plausibility_EventsAdapt_active-passive.png"
ggsave(paste(savedir,savename,sep="/"), width=20, height=8, units='cm')

###Attempt with human ceiling

dataset = "EventsAdapt"
plot_data = merged %>% filter(Dataset==dataset, TrialType=="normal",Plot!="HumanCeiling")

ceil_data = merged %>% filter(TrialType=="normal",Plot=="HumanCeiling")
ceil_summary <- ceil_data %>%
  group_by(VoiceType) %>%
  summarise(
    MeanAccuracy = mean(Accuracy),
    sd = sd(Accuracy),
    n = n(),
    se = sd / sqrt(n)
  )

voice_types = c(ceil_summary$VoiceType)

ceil_data = ceil_data %>%
  mutate(MeanAccuracy = NA) %>%
  mutate(SE = NA)

for( vt in voice_types){
  ceil_data = ceil_data %>%
    mutate(MeanAccuracy = ifelse(ceil_data$VoiceType==vt, subset(ceil_summary, VoiceType == vt)$MeanAccuracy, ceil_data$MeanAccuracy)) %>%
    mutate(SE = ifelse(ceil_data$VoiceType==vt, subset(ceil_summary, VoiceType == vt)$se, ceil_data$SE))
}

ggplot(data = plot_data,
       mapping = aes(x=Layer, y=Accuracy, color=VoiceType, group=VoiceType))+
  facet_grid(~Model, scales="free_x")+
  geom_hline(yintercept=.5, linetype='dotted')+
  geom_hline(yintercept=1, linetype='dotted')+
  stat_summary(geom='line', fun='mean')+
  stat_summary(geom='errorbar', fun.data='mean_se',
               size = 0.2, width=0.1)+
  geom_point(inherit.aes=FALSE, data = ceil_data, aes(x=Layer, y=MeanAccuracy, color=VoiceType), size=0.2)+
  geom_errorbar(inherit.aes=FALSE, data = ceil_data, aes(x=Layer, ymin=MeanAccuracy-SE, ymax=MeanAccuracy+SE, color=VoiceType),width = 0.05) +
    geom_text(data = ceil_data, group=ceil_data$VoiceType, x =ifelse(ceil_data$Model=="gpt2-xl", ceil_data$Layer - 6, ceil_data$Layer - 3),  y = 0.98,
            size = 2.5,
           label = "ceiling", 
           colour = "#6d6d6d") +
  theme_classic()+
  scale_x_continuous(breaks=breaks_fun)+
  scale_y_continuous(breaks=seq(0.2,1,0.1))


savename <- "model2plausibility_EventsAdapt_active-passive_ceiled.png"
ggsave(paste(savedir,savename,sep="/"), width=20, height=8, units='cm', device='tiff', dpi=700)

TrialTypes

dataset = "EventsAdapt"

ggplot(data = dat %>% filter(Dataset==dataset, VoiceType=="active-active", !grepl('AAR', TrialType)),
       mapping = aes(x=Layer, y=Accuracy, color=TrialType, group=TrialType))+
  facet_wrap(~Model, scales="free_x")+
  scale_x_continuous(breaks = breaks_fun, limits = c(0, NA)) + 
  geom_hline(yintercept=.5, linetype='dotted')+
  geom_hline(yintercept=1, linetype='dotted')+
  stat_summary(geom='line', fun='mean')+
  stat_summary(geom='errorbar', fun.data='mean_se',
               size = 0.2, width=0)+
  theme_classic()

savename <- "model2plausibility_EventsAdapt_AI-AAN.png"
ggsave(paste(savedir,savename,sep="/"), width=20, height=8, units='cm')

###Attempt with human ceiling

dataset = "EventsAdapt"
plot_data = merged %>% filter(Dataset==dataset, VoiceType=="active-active", !grepl('AAR', TrialType),Plot!="HumanCeiling")
ceil_data = merged %>% filter(VoiceType=="active-active",Plot=="HumanCeiling",!grepl('AAR', TrialType))

ceil_summary <- ceil_data %>%
  group_by(TrialType) %>%
  summarise(
    MeanAccuracy = mean(Accuracy),
    sd = sd(Accuracy),
    n = n(),
    se = sd / sqrt(n)
  )

trial_types = c(ceil_summary$TrialType)

ceil_data = ceil_data %>%
  mutate(MeanAccuracy = NA) %>%
  mutate(SE = NA)

for( tt in trial_types){
  ceil_data = ceil_data %>%
    mutate(MeanAccuracy = ifelse(ceil_data$TrialType==tt, subset(ceil_summary, TrialType == tt)$MeanAccuracy, ceil_data$MeanAccuracy)) %>%
    mutate(SE = ifelse(ceil_data$TrialType==tt, subset(ceil_summary, TrialType == tt)$se, ceil_data$SE))
}

ggplot(data = plot_data,
       mapping = aes(x=Layer, y=Accuracy, color=TrialType, group=TrialType))+
  facet_grid(~Model, scales="free_x")+
  geom_hline(yintercept=.5, linetype='dotted')+
  geom_hline(yintercept=1, linetype='dotted')+
  stat_summary(geom='line', fun='mean')+
  stat_summary(geom='errorbar', fun.data='mean_se',
               size = 0.2, width=0.1)+
  geom_point(data = ceil_data, aes(x=Layer, y=MeanAccuracy, color=TrialType), size=0.2)+
  geom_errorbar(data = ceil_data, aes(x=Layer, ymin=MeanAccuracy-SE, ymax=MeanAccuracy+SE, color=TrialType), width = 0.05) +
  geom_text(data = ceil_data, group=ceil_data$TrialType, x =ifelse(ceil_data$Model=="gpt2-xl", ceil_data$Layer - 6, ceil_data$Layer - 3),  y = 0.98,
        size = 2.5,
       label = "ceiling", 
       colour = "#6d6d6d") +
  theme_classic()+
  scale_x_continuous(breaks=breaks_fun)+
  scale_y_continuous(breaks=seq(0.5,1,0.1))


savename <- "model2plausibility_EventsAdapt_AI-AAN_ceiled.png"
ggsave(paste(savedir,savename,sep="/"), width=20, height=8, units='cm', device='tiff', dpi=700)
LS0tCnRpdGxlOiAiUHJvYmluZyByZXN1bHRzIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgojIFNFVFVQCgpgYGB7ciBzZXR1cCwgaW5jbHVkZT1GQUxTRSwgZWNobz1UUlVFfQprbml0cjo6b3B0c19jaHVuayRzZXQoZWNobyA9IFRSVUUpCnJtKGxpc3Q9bHMoKSkgCmxpYnJhcnkoZ2dwbG90MikKbGlicmFyeShkcGx5cikKbGlicmFyeSh0aWR5cikKbGlicmFyeShzdHJpbmdyKQpsaWJyYXJ5KGdyaWQpCmxpYnJhcnkoZ3JpZEV4dHJhKQpsaWJyYXJ5KG9wZXJhdG9yLnRvb2xzKQpsaWJyYXJ5KGxtZTQpCmxpYnJhcnkobG1lclRlc3QpCmxpYnJhcnkoY29jb3IpCmxpYnJhcnkocGF0Y2h3b3JrKQpsaWJyYXJ5KGd0b29scykKCnNvdXJjZSgnZGF0YWxvYWRlcl91dGlscy5SJykgI2luY2x1ZGVzIG5vcm1hbGl6YXRpb25zLCByZWFkX2RhdGEgZnVuY3Rpb25zCnNvdXJjZSgnc3RhdHNfdXRpbHMuUicpCgojIFN1cHByZXNzIHN1bW1hcmlzZSBpbmZvCm9wdGlvbnMoZHBseXIuc3VtbWFyaXNlLmluZm9ybSA9IEZBTFNFKQpgYGAKCiMjIE91dHB1dCBkaXIKYGBge3J9CnBhdGggPC0gcGFzdGUoInJlc3VsdHMvIikKaWZlbHNlKCFkaXIuZXhpc3RzKHBhdGgpLCBkaXIuY3JlYXRlKHBhdGgpLCBGQUxTRSkKc2F2ZWRpciA8LSBwYXN0ZShwYXRoLCJQcm9iaW5nIixzZXA9JycpCmlmZWxzZSghZGlyLmV4aXN0cyhzYXZlZGlyKSwgZGlyLmNyZWF0ZShzYXZlZGlyKSwgRkFMU0UpCmBgYAoKCiMgUkVBRCBEQVRBCmBgYHtyfQptb2RlbDJwbGF1c2liaWxpdHlfZGlyID0gIi4uL3Byb2JpbmcvcmVzdWx0cy9tb2RlbDJwbGF1c2liaWxpdHkiCmRhdGFmaWxlID0gImFsbF9jb21iaW5lZF9rZm9sZF9uZXcuY3N2IgoKZGF0ID0gcmVhZC5jc3YocGFzdGUobW9kZWwycGxhdXNpYmlsaXR5X2RpciwgZGF0YWZpbGUsIHNlcD0iLyIpKQoKCmRhdCA8LSBzdWJzZXQoZGF0LCBzZWxlY3QgPSAtWCkKCmRhdCRMYXllciA9IGFzLmZhY3RvcihkYXQkTGF5ZXIpCmRhdCRNb2RlbCA9IGZhY3RvcihkYXQkTW9kZWwsIGxldmVscz1jKCJiZXJ0LWxhcmdlLWNhc2VkIiwgInJvYmVydGEtbGFyZ2UiLCAiZ3B0Mi14bCIpKQpgYGAKCiMgU1RBVFMKCmBgYHtyfQpkYXQuRFRGaXQgPSBkYXQgJT4lIGZpbHRlcihEYXRhc2V0PT0iRFRGaXQiKQoKbS5EVEZpdCA9IGxtKEFjY3VyYWN5fkxheWVyOk1vZGVsLCBkYXRhPWRhdC5EVEZpdCkKc3VtbWFyeShtLkRURml0KQpgYGAKCiMgUkVBRCBIVU1BTiBDRUlMSU5HIERBVEEKYGBge3J9CmRhdCRMYXllciA9IGFzLm51bWVyaWMoYXMuY2hhcmFjdGVyKGRhdCRMYXllcikpCgoKbW9kZWwycGxhdXNpYmlsaXR5X2RpciA9ICIuLi9wcm9iaW5nL3Jlc3VsdHMvbW9kZWwyaHVtYW5fY2VpbGluZyIKCnJlYWRfZGF0YSA8LSBmdW5jdGlvbihkaXJlY3RvcnksIGZpbGVuYW1lKSB7CiAgcHJpbnQocGFzdGUoZmlsZW5hbWUpKQogIG1ldGFkYXRhID0gc3RyX3NwbGl0KGZpbGVuYW1lLCAiXyIpW1sxXV0KICBkID0gcmVhZC5kZWxpbShwYXN0ZShtb2RlbDJwbGF1c2liaWxpdHlfZGlyLCBmaWxlbmFtZSwgc2VwPScvJyksIAogICAgICAgICAgICAgICAgIGhlYWRlcj1GQUxTRSwgc2VwPScsJykKICBkID0gZCAlPiUKICAgIG11dGF0ZShEYXRhc2V0ID0gbWV0YWRhdGFbWzJdXSkgJT4lCiAgICBtdXRhdGUoVm9pY2VUeXBlID0gbWV0YWRhdGFbWzNdXSkgJT4lCiAgICBtdXRhdGUoVHJpYWxUeXBlID0gbWV0YWRhdGFbWzRdXSkgJT4lCiAgICBtdXRhdGUoVHJpYWxUeXBlID0gc3RyX3JlcGxhY2UoVHJpYWxUeXBlLCAiLmNzdiIsICIiKSkgJT4lCiAgICByZW5hbWUoSXRlcmF0aW9uID0gVjEpICAlPiUKICAgIHJlbmFtZShBY2N1cmFjeSA9IFYyKSAgJT4lCiAgICBtdXRhdGUoUGxvdCA9ICJIdW1hbkNlaWxpbmciKQp9CgpmaWxlbmFtZXMgPSBsaXN0LmZpbGVzKHBhdGg9bW9kZWwycGxhdXNpYmlsaXR5X2RpciwgcGF0dGVybj0nKi5jc3YnKQpkYXQuY2VpbGluZyA9IGRvLmNhbGwocmJpbmQsIGxhcHBseShmaWxlbmFtZXMsIGZ1bmN0aW9uKHgpIHJlYWRfZGF0YShtb2RlbDJwbGF1c2liaWxpdHlfZGlyLCB4KSkpCgphZGRfaW5mbyA8LSBmdW5jdGlvbihkYXRhZnJhbWUsIG1vZGVsX25hbWUpIHsKICBpZiAoZ3JlcGwoImdwdDIteGwiLG1vZGVsX25hbWUpID09IEZBTFNFKXsKICAgIHRvYWRkID0gNQogIH1lbHNlewogICAgdG9hZGQgPSAxMAogIH0KICBkYXRhZnJhbWUgPSBkYXRhZnJhbWUgJT4lCiAgICBtdXRhdGUoTGF5ZXIgPSBtYXgoc3Vic2V0KGRhdCwgTW9kZWw9PW1vZGVsX25hbWUpJExheWVyKSArIHRvYWRkKSAlPiUKICAgIG11dGF0ZShNb2RlbCA9IG1vZGVsX25hbWUpCn0KCm1vZGVscyA9IHVuaXF1ZShjKGRhdCRNb2RlbCkpCiAgCmRhdC5jZWlsaW5nLmZ1bGwgPSBkby5jYWxsKHJiaW5kLCBsYXBwbHkobW9kZWxzLCBmdW5jdGlvbih4KSBhZGRfaW5mbyhkYXQuY2VpbGluZywgeCkpKQpgYGAKYGBge3J9Cm1lcmdlZCA8LSByYmluZChkYXQsIGRhdC5jZWlsaW5nLmZ1bGwpCmBgYAoKIyBQTE9UCgojIyBicmVha3MgZnVuY3Rpb24geCBheGlzCmBgYHtyfQpicmVha3NfZnVuIDwtIGZ1bmN0aW9uKHgpIHsKICBpZiAobWF4KHgpID4gNDApIHsKICAgIGMoc2VxKDAsIDUwLCAxMCksIG1heCh4KSArIDEwKQogIH0gZWxzZSB7CiAgICBjKHNlcSgwLCAyNSwgNSksIG1heCh4KSArIDUpCiAgfQp9CgpgYGAKCiMjIEFsbCBkYXRhc2V0cwpgYGB7ciwgZmlnLndpZHRoPTE1LCBmaWcuaGVpZ2h0PTh9CnBsb3RfZGF0YSA9IG1lcmdlZCAlPiUgZmlsdGVyKFRyaWFsVHlwZT09Im5vcm1hbCIsIFZvaWNlVHlwZT09Im5vcm1hbCIsUGxvdCE9Ikh1bWFuQ2VpbGluZyIpCnBsb3RfZGF0YSRMYXllciA9IGFzLm51bWVyaWMoYXMuY2hhcmFjdGVyKHBsb3RfZGF0YSRMYXllcikpCgpjZWlsX2RhdGEgPSBtZXJnZWQgJT4lIGZpbHRlcihUcmlhbFR5cGU9PSJub3JtYWwiLCBWb2ljZVR5cGU9PSJub3JtYWwiLFBsb3Q9PSJIdW1hbkNlaWxpbmciKQpjZWlsX3N1bW1hcnkgPC0gY2VpbF9kYXRhICU+JQogIGdyb3VwX2J5KERhdGFzZXQpICU+JQogIHN1bW1hcmlzZSgKICAgIE1lYW5BY2N1cmFjeSA9IG1lYW4oQWNjdXJhY3kpLAogICAgc2QgPSBzZChBY2N1cmFjeSksCiAgICBuID0gbigpLAogICAgU0UgPSBzZCAvIHNxcnQobikKICApCgpkYXRhc2V0cyA9IGMoY2VpbF9zdW1tYXJ5JERhdGFzZXQpCgpjZWlsX2RhdGEgPSBjZWlsX2RhdGEgJT4lIAogIG11dGF0ZShNZWFuQWNjdXJhY3kgPSBOQSkgJT4lCiAgbXV0YXRlKFNFID0gTkEpCgpmb3IoIGR0IGluIGRhdGFzZXRzKXsKICBjZWlsX2RhdGEgPSBjZWlsX2RhdGEgJT4lCiAgICBtdXRhdGUoTWVhbkFjY3VyYWN5ID0gaWZlbHNlKGNlaWxfZGF0YSREYXRhc2V0PT1kdCwgc3Vic2V0KGNlaWxfc3VtbWFyeSwgRGF0YXNldCA9PSBkdCkkTWVhbkFjY3VyYWN5LCBjZWlsX2RhdGEkTWVhbkFjY3VyYWN5KSkgJT4lCiAgICBtdXRhdGUoU0UgPSBpZmVsc2UoY2VpbF9kYXRhJERhdGFzZXQ9PWR0LCBzdWJzZXQoY2VpbF9zdW1tYXJ5LCBEYXRhc2V0ID09IGR0KSRTRSwgY2VpbF9kYXRhJFNFKSkKfQoKZ2dwbG90KGRhdGEgPSBwbG90X2RhdGEsCiAgICAgICBtYXBwaW5nID0gYWVzKHg9TGF5ZXIsIHk9QWNjdXJhY3ksIGNvbG9yPURhdGFzZXQsIGdyb3VwID0gRGF0YXNldCkpKwogIGZhY2V0X2dyaWQofk1vZGVsLCBzY2FsZXM9ImZyZWVfeCIpKwogIGdlb21faGxpbmUoeWludGVyY2VwdD0uNSwgbGluZXR5cGU9J2RvdHRlZCcpKwogIGdlb21faGxpbmUoeWludGVyY2VwdD0xLCBsaW5ldHlwZT0nZG90dGVkJykrCiAgc3RhdF9zdW1tYXJ5KGdlb209J2xpbmUnLCBmdW49J21lYW4nKSsKICBzdGF0X3N1bW1hcnkoZ2VvbT0nZXJyb3JiYXInLCBmdW4uZGF0YT0nbWVhbl9zZScsCiAgICAgICAgICAgICAgIHNpemUgPSAwLjIsIHdpZHRoPTAuMSkrCiAgZ2VvbV9wb2ludChkYXRhID0gY2VpbF9kYXRhLCBhZXMoeD1MYXllciwgeT1NZWFuQWNjdXJhY3ksIGNvbG9yPURhdGFzZXQpLCBzaXplPTAuMikrCiAgZ2VvbV9lcnJvcmJhcihkYXRhID0gY2VpbF9kYXRhLCBhZXMoeD1MYXllciwgeW1pbj1NZWFuQWNjdXJhY3ktU0UsIHltYXg9TWVhbkFjY3VyYWN5K1NFLCBjb2xvcj1EYXRhc2V0KSx3aWR0aCA9IDAuMDUpICsKICBnZW9tX3RleHQoZGF0YSA9IGNlaWxfZGF0YSwgZ3JvdXA9Y2VpbF9kYXRhJERhdGFzZXQsIHggPWlmZWxzZShjZWlsX2RhdGEkTW9kZWw9PSJncHQyLXhsIiwgY2VpbF9kYXRhJExheWVyIC0gNiwgY2VpbF9kYXRhJExheWVyIC0gMyksICB5ID0gMC45OCwKICAgICAgICAgICAgc2l6ZSA9IDIuNSwKICAgICAgICAgICAgbGFiZWwgPSAiY2VpbGluZyIsIAogICAgICAgICAgICBjb2xvdXIgPSAiIzZkNmQ2ZCIpICsKICB0aGVtZV9jbGFzc2ljKCkrCiAgc2NhbGVfeF9jb250aW51b3VzKGJyZWFrcz1icmVha3NfZnVuKSsKICBzY2FsZV95X2NvbnRpbnVvdXMoYnJlYWtzPXNlcSgwLjQsMS4yLDAuMSkpCgpzYXZlbmFtZSA8LSAibW9kZWwycGxhdXNpYmlsaXR5X2FsbF9kYXRhc2V0c19jZWlsZWQucG5nIgpnZ3NhdmUocGFzdGUoc2F2ZWRpcixzYXZlbmFtZSxzZXA9Ii8iKSwgd2lkdGg9MjAsIGhlaWdodD04LCB1bml0cz0nY20nLCBkZXZpY2U9J3RpZmYnLCBkcGk9NzAwKQpgYGAKCiMjIERURml0CmBgYHtyfQpkYXRhc2V0ID0gIkRURml0IgoKZ2dwbG90KGRhdGEgPSBkYXQgJT4lIGZpbHRlcihEYXRhc2V0PT1kYXRhc2V0KSwKICAgICAgIG1hcHBpbmcgPSBhZXMoeD1MYXllciwgeT1BY2N1cmFjeSwgZ3JvdXA9ZGF0YXNldCkpKwogIGZhY2V0X3dyYXAoflBsb3QsIHNjYWxlcz0iZnJlZV94IikrCiAgc2NhbGVfeF9jb250aW51b3VzKGJyZWFrcyA9IGJyZWFrc19mdW4sIGxpbWl0cyA9IGMoMCwgTkEpKSArIAogIGdlb21faGxpbmUoeWludGVyY2VwdD0uNSwgbGluZXR5cGU9J2RvdHRlZCcpKwogIGdlb21faGxpbmUoeWludGVyY2VwdD0xLCBsaW5ldHlwZT0nZG90dGVkJykrCiAgc3RhdF9zdW1tYXJ5KGdlb209J2xpbmUnLCBmdW49J21lYW4nKSsKICBzdGF0X3N1bW1hcnkoZ2VvbT0nZXJyb3JiYXInLCBmdW4uZGF0YT0nbWVhbl9zZScsCiAgICAgICAgICAgICAgIGNvbG9yID0gJ2JsYWNrJywgc2l6ZSA9IDAuNSwgd2lkdGg9MC4xKSsKICB0aGVtZV9jbGFzc2ljKCkKCnNhdmVuYW1lIDwtICJtb2RlbDJwbGF1c2liaWxpdHlfRFRGaXQucG5nIgpnZ3NhdmUocGFzdGUoc2F2ZWRpcixzYXZlbmFtZSxzZXA9Ii8iKSwgd2lkdGg9MTYsIGhlaWdodD04LCB1bml0cz0nY20nKQpgYGAKCiMjIEV2ZW50c1JldgpgYGB7cn0KZGF0YXNldCA9ICJFdmVudHNSZXYiCgpnZ3Bsb3QoZGF0YSA9IGRhdCAlPiUgZmlsdGVyKERhdGFzZXQ9PWRhdGFzZXQpLAogICAgICAgbWFwcGluZyA9IGFlcyh4PUxheWVyLCB5PUFjY3VyYWN5LCBncm91cD1kYXRhc2V0KSkrCiAgZmFjZXRfZ3JpZCh+UGxvdCwgc2NhbGVzPSJmcmVlX3giKSsKICBzY2FsZV94X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX2Z1biwgbGltaXRzID0gYygwLCBOQSkpICsgCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0PS41LCBsaW5ldHlwZT0nZG90dGVkJykrCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0PTEsIGxpbmV0eXBlPSdkb3R0ZWQnKSsKICBzdGF0X3N1bW1hcnkoZ2VvbT0nbGluZScsIGZ1bj0nbWVhbicpKwogIHN0YXRfc3VtbWFyeShnZW9tPSdlcnJvcmJhcicsIGZ1bi5kYXRhPSdtZWFuX3NlJywKICAgICAgICAgICAgICAgY29sb3IgPSAnYmxhY2snLCBzaXplID0gMC41LCB3aWR0aD0wLjEpKwogIHRoZW1lX2NsYXNzaWMoKQoKc2F2ZW5hbWUgPC0gIm1vZGVsMnBsYXVzaWJpbGl0eV9FdmVudHNSZXYucG5nIgpnZ3NhdmUocGFzdGUoc2F2ZWRpcixzYXZlbmFtZSxzZXA9Ii8iKSwgd2lkdGg9MTYsIGhlaWdodD04LCB1bml0cz0nY20nKQpgYGAKCiMjIEV2ZW50c0FkYXB0CgojIyMgQWN0aXZlIFBhc3NpdmUKYGBge3J9CmRhdGFzZXQgPSAiRXZlbnRzQWRhcHQiCgpnZ3Bsb3QoZGF0YSA9IGRhdCAlPiUgZmlsdGVyKERhdGFzZXQ9PWRhdGFzZXQsIFRyaWFsVHlwZT09Im5vcm1hbCIpLAogICAgICAgbWFwcGluZyA9IGFlcyh4PUxheWVyLCB5PUFjY3VyYWN5LCBjb2xvcj1Wb2ljZVR5cGUsIGdyb3VwPVZvaWNlVHlwZSkpKwogIGZhY2V0X2dyaWQofk1vZGVsLCBzY2FsZXM9ImZyZWVfeCIpKyAKICBzY2FsZV94X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX2Z1biwgbGltaXRzID0gYygwLCBOQSkpICsgCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0PS41LCBsaW5ldHlwZT0nZG90dGVkJykrCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0PTEsIGxpbmV0eXBlPSdkb3R0ZWQnKSsKICBzdGF0X3N1bW1hcnkoZ2VvbT0nbGluZScsIGZ1bj0nbWVhbicpKwogIHN0YXRfc3VtbWFyeShnZW9tPSdlcnJvcmJhcicsIGZ1bi5kYXRhPSdtZWFuX3NlJywKICAgICAgICAgICAgICAgc2l6ZSA9IDAuMiwgd2lkdGg9MCkrCiAgdGhlbWVfY2xhc3NpYygpCgpzYXZlbmFtZSA8LSAibW9kZWwycGxhdXNpYmlsaXR5X0V2ZW50c0FkYXB0X2FjdGl2ZS1wYXNzaXZlLnBuZyIKZ2dzYXZlKHBhc3RlKHNhdmVkaXIsc2F2ZW5hbWUsc2VwPSIvIiksIHdpZHRoPTIwLCBoZWlnaHQ9OCwgdW5pdHM9J2NtJykKYGBgCgojIyNBdHRlbXB0IHdpdGggaHVtYW4gY2VpbGluZwpgYGB7ciwgZmlnLndpZHRoPTE1LCBmaWcuaGVpZ2h0PTh9CmRhdGFzZXQgPSAiRXZlbnRzQWRhcHQiCnBsb3RfZGF0YSA9IG1lcmdlZCAlPiUgZmlsdGVyKERhdGFzZXQ9PWRhdGFzZXQsIFRyaWFsVHlwZT09Im5vcm1hbCIsUGxvdCE9Ikh1bWFuQ2VpbGluZyIpCgpjZWlsX2RhdGEgPSBtZXJnZWQgJT4lIGZpbHRlcihUcmlhbFR5cGU9PSJub3JtYWwiLFBsb3Q9PSJIdW1hbkNlaWxpbmciKQpjZWlsX3N1bW1hcnkgPC0gY2VpbF9kYXRhICU+JQogIGdyb3VwX2J5KFZvaWNlVHlwZSkgJT4lCiAgc3VtbWFyaXNlKAogICAgTWVhbkFjY3VyYWN5ID0gbWVhbihBY2N1cmFjeSksCiAgICBzZCA9IHNkKEFjY3VyYWN5KSwKICAgIG4gPSBuKCksCiAgICBzZSA9IHNkIC8gc3FydChuKQogICkKCnZvaWNlX3R5cGVzID0gYyhjZWlsX3N1bW1hcnkkVm9pY2VUeXBlKQoKY2VpbF9kYXRhID0gY2VpbF9kYXRhICU+JQogIG11dGF0ZShNZWFuQWNjdXJhY3kgPSBOQSkgJT4lCiAgbXV0YXRlKFNFID0gTkEpCgpmb3IoIHZ0IGluIHZvaWNlX3R5cGVzKXsKICBjZWlsX2RhdGEgPSBjZWlsX2RhdGEgJT4lCiAgICBtdXRhdGUoTWVhbkFjY3VyYWN5ID0gaWZlbHNlKGNlaWxfZGF0YSRWb2ljZVR5cGU9PXZ0LCBzdWJzZXQoY2VpbF9zdW1tYXJ5LCBWb2ljZVR5cGUgPT0gdnQpJE1lYW5BY2N1cmFjeSwgY2VpbF9kYXRhJE1lYW5BY2N1cmFjeSkpICU+JQogICAgbXV0YXRlKFNFID0gaWZlbHNlKGNlaWxfZGF0YSRWb2ljZVR5cGU9PXZ0LCBzdWJzZXQoY2VpbF9zdW1tYXJ5LCBWb2ljZVR5cGUgPT0gdnQpJHNlLCBjZWlsX2RhdGEkU0UpKQp9CgpnZ3Bsb3QoZGF0YSA9IHBsb3RfZGF0YSwKICAgICAgIG1hcHBpbmcgPSBhZXMoeD1MYXllciwgeT1BY2N1cmFjeSwgY29sb3I9Vm9pY2VUeXBlLCBncm91cD1Wb2ljZVR5cGUpKSsKICBmYWNldF9ncmlkKH5Nb2RlbCwgc2NhbGVzPSJmcmVlX3giKSsKICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQ9LjUsIGxpbmV0eXBlPSdkb3R0ZWQnKSsKICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQ9MSwgbGluZXR5cGU9J2RvdHRlZCcpKwogIHN0YXRfc3VtbWFyeShnZW9tPSdsaW5lJywgZnVuPSdtZWFuJykrCiAgc3RhdF9zdW1tYXJ5KGdlb209J2Vycm9yYmFyJywgZnVuLmRhdGE9J21lYW5fc2UnLAogICAgICAgICAgICAgICBzaXplID0gMC4yLCB3aWR0aD0wLjEpKwogIGdlb21fcG9pbnQoaW5oZXJpdC5hZXM9RkFMU0UsIGRhdGEgPSBjZWlsX2RhdGEsIGFlcyh4PUxheWVyLCB5PU1lYW5BY2N1cmFjeSwgY29sb3I9Vm9pY2VUeXBlKSwgc2l6ZT0wLjIpKwogIGdlb21fZXJyb3JiYXIoaW5oZXJpdC5hZXM9RkFMU0UsIGRhdGEgPSBjZWlsX2RhdGEsIGFlcyh4PUxheWVyLCB5bWluPU1lYW5BY2N1cmFjeS1TRSwgeW1heD1NZWFuQWNjdXJhY3krU0UsIGNvbG9yPVZvaWNlVHlwZSksd2lkdGggPSAwLjA1KSArCiAgICBnZW9tX3RleHQoZGF0YSA9IGNlaWxfZGF0YSwgZ3JvdXA9Y2VpbF9kYXRhJFZvaWNlVHlwZSwgeCA9aWZlbHNlKGNlaWxfZGF0YSRNb2RlbD09ImdwdDIteGwiLCBjZWlsX2RhdGEkTGF5ZXIgLSA2LCBjZWlsX2RhdGEkTGF5ZXIgLSAzKSwgIHkgPSAwLjk4LAogICAgICAgICAgICBzaXplID0gMi41LAogICAgICAgICAgIGxhYmVsID0gImNlaWxpbmciLCAKICAgICAgICAgICBjb2xvdXIgPSAiIzZkNmQ2ZCIpICsKICB0aGVtZV9jbGFzc2ljKCkrCiAgc2NhbGVfeF9jb250aW51b3VzKGJyZWFrcz1icmVha3NfZnVuKSsKICBzY2FsZV95X2NvbnRpbnVvdXMoYnJlYWtzPXNlcSgwLjIsMSwwLjEpKQoKc2F2ZW5hbWUgPC0gIm1vZGVsMnBsYXVzaWJpbGl0eV9FdmVudHNBZGFwdF9hY3RpdmUtcGFzc2l2ZV9jZWlsZWQucG5nIgpnZ3NhdmUocGFzdGUoc2F2ZWRpcixzYXZlbmFtZSxzZXA9Ii8iKSwgd2lkdGg9MjAsIGhlaWdodD04LCB1bml0cz0nY20nLCBkZXZpY2U9J3RpZmYnLCBkcGk9NzAwKQpgYGAKCiMjIyBUcmlhbFR5cGVzCmBgYHtyfQpkYXRhc2V0ID0gIkV2ZW50c0FkYXB0IgoKZ2dwbG90KGRhdGEgPSBkYXQgJT4lIGZpbHRlcihEYXRhc2V0PT1kYXRhc2V0LCBWb2ljZVR5cGU9PSJhY3RpdmUtYWN0aXZlIiwgIWdyZXBsKCdBQVInLCBUcmlhbFR5cGUpKSwKICAgICAgIG1hcHBpbmcgPSBhZXMoeD1MYXllciwgeT1BY2N1cmFjeSwgY29sb3I9VHJpYWxUeXBlLCBncm91cD1UcmlhbFR5cGUpKSsKICBmYWNldF93cmFwKH5Nb2RlbCwgc2NhbGVzPSJmcmVlX3giKSsKICBzY2FsZV94X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX2Z1biwgbGltaXRzID0gYygwLCBOQSkpICsgCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0PS41LCBsaW5ldHlwZT0nZG90dGVkJykrCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0PTEsIGxpbmV0eXBlPSdkb3R0ZWQnKSsKICBzdGF0X3N1bW1hcnkoZ2VvbT0nbGluZScsIGZ1bj0nbWVhbicpKwogIHN0YXRfc3VtbWFyeShnZW9tPSdlcnJvcmJhcicsIGZ1bi5kYXRhPSdtZWFuX3NlJywKICAgICAgICAgICAgICAgc2l6ZSA9IDAuMiwgd2lkdGg9MCkrCiAgdGhlbWVfY2xhc3NpYygpCgpzYXZlbmFtZSA8LSAibW9kZWwycGxhdXNpYmlsaXR5X0V2ZW50c0FkYXB0X0FJLUFBTi5wbmciCmdnc2F2ZShwYXN0ZShzYXZlZGlyLHNhdmVuYW1lLHNlcD0iLyIpLCB3aWR0aD0yMCwgaGVpZ2h0PTgsIHVuaXRzPSdjbScpCmBgYAoKIyMjQXR0ZW1wdCB3aXRoIGh1bWFuIGNlaWxpbmcKYGBge3IsIGZpZy53aWR0aD0xNSwgZmlnLmhlaWdodD04fQpkYXRhc2V0ID0gIkV2ZW50c0FkYXB0IgpwbG90X2RhdGEgPSBtZXJnZWQgJT4lIGZpbHRlcihEYXRhc2V0PT1kYXRhc2V0LCBWb2ljZVR5cGU9PSJhY3RpdmUtYWN0aXZlIiwgIWdyZXBsKCdBQVInLCBUcmlhbFR5cGUpLFBsb3QhPSJIdW1hbkNlaWxpbmciKQpjZWlsX2RhdGEgPSBtZXJnZWQgJT4lIGZpbHRlcihWb2ljZVR5cGU9PSJhY3RpdmUtYWN0aXZlIixQbG90PT0iSHVtYW5DZWlsaW5nIiwhZ3JlcGwoJ0FBUicsIFRyaWFsVHlwZSkpCgpjZWlsX3N1bW1hcnkgPC0gY2VpbF9kYXRhICU+JQogIGdyb3VwX2J5KFRyaWFsVHlwZSkgJT4lCiAgc3VtbWFyaXNlKAogICAgTWVhbkFjY3VyYWN5ID0gbWVhbihBY2N1cmFjeSksCiAgICBzZCA9IHNkKEFjY3VyYWN5KSwKICAgIG4gPSBuKCksCiAgICBzZSA9IHNkIC8gc3FydChuKQogICkKCnRyaWFsX3R5cGVzID0gYyhjZWlsX3N1bW1hcnkkVHJpYWxUeXBlKQoKY2VpbF9kYXRhID0gY2VpbF9kYXRhICU+JQogIG11dGF0ZShNZWFuQWNjdXJhY3kgPSBOQSkgJT4lCiAgbXV0YXRlKFNFID0gTkEpCgpmb3IoIHR0IGluIHRyaWFsX3R5cGVzKXsKICBjZWlsX2RhdGEgPSBjZWlsX2RhdGEgJT4lCiAgICBtdXRhdGUoTWVhbkFjY3VyYWN5ID0gaWZlbHNlKGNlaWxfZGF0YSRUcmlhbFR5cGU9PXR0LCBzdWJzZXQoY2VpbF9zdW1tYXJ5LCBUcmlhbFR5cGUgPT0gdHQpJE1lYW5BY2N1cmFjeSwgY2VpbF9kYXRhJE1lYW5BY2N1cmFjeSkpICU+JQogICAgbXV0YXRlKFNFID0gaWZlbHNlKGNlaWxfZGF0YSRUcmlhbFR5cGU9PXR0LCBzdWJzZXQoY2VpbF9zdW1tYXJ5LCBUcmlhbFR5cGUgPT0gdHQpJHNlLCBjZWlsX2RhdGEkU0UpKQp9CgpnZ3Bsb3QoZGF0YSA9IHBsb3RfZGF0YSwKICAgICAgIG1hcHBpbmcgPSBhZXMoeD1MYXllciwgeT1BY2N1cmFjeSwgY29sb3I9VHJpYWxUeXBlLCBncm91cD1UcmlhbFR5cGUpKSsKICBmYWNldF9ncmlkKH5Nb2RlbCwgc2NhbGVzPSJmcmVlX3giKSsKICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQ9LjUsIGxpbmV0eXBlPSdkb3R0ZWQnKSsKICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQ9MSwgbGluZXR5cGU9J2RvdHRlZCcpKwogIHN0YXRfc3VtbWFyeShnZW9tPSdsaW5lJywgZnVuPSdtZWFuJykrCiAgc3RhdF9zdW1tYXJ5KGdlb209J2Vycm9yYmFyJywgZnVuLmRhdGE9J21lYW5fc2UnLAogICAgICAgICAgICAgICBzaXplID0gMC4yLCB3aWR0aD0wLjEpKwogIGdlb21fcG9pbnQoZGF0YSA9IGNlaWxfZGF0YSwgYWVzKHg9TGF5ZXIsIHk9TWVhbkFjY3VyYWN5LCBjb2xvcj1UcmlhbFR5cGUpLCBzaXplPTAuMikrCiAgZ2VvbV9lcnJvcmJhcihkYXRhID0gY2VpbF9kYXRhLCBhZXMoeD1MYXllciwgeW1pbj1NZWFuQWNjdXJhY3ktU0UsIHltYXg9TWVhbkFjY3VyYWN5K1NFLCBjb2xvcj1UcmlhbFR5cGUpLCB3aWR0aCA9IDAuMDUpICsKICBnZW9tX3RleHQoZGF0YSA9IGNlaWxfZGF0YSwgZ3JvdXA9Y2VpbF9kYXRhJFRyaWFsVHlwZSwgeCA9aWZlbHNlKGNlaWxfZGF0YSRNb2RlbD09ImdwdDIteGwiLCBjZWlsX2RhdGEkTGF5ZXIgLSA2LCBjZWlsX2RhdGEkTGF5ZXIgLSAzKSwgIHkgPSAwLjk4LAogICAgICAgIHNpemUgPSAyLjUsCiAgICAgICBsYWJlbCA9ICJjZWlsaW5nIiwgCiAgICAgICBjb2xvdXIgPSAiIzZkNmQ2ZCIpICsKICB0aGVtZV9jbGFzc2ljKCkrCiAgc2NhbGVfeF9jb250aW51b3VzKGJyZWFrcz1icmVha3NfZnVuKSsKICBzY2FsZV95X2NvbnRpbnVvdXMoYnJlYWtzPXNlcSgwLjUsMSwwLjEpKQoKc2F2ZW5hbWUgPC0gIm1vZGVsMnBsYXVzaWJpbGl0eV9FdmVudHNBZGFwdF9BSS1BQU5fY2VpbGVkLnBuZyIKZ2dzYXZlKHBhc3RlKHNhdmVkaXIsc2F2ZW5hbWUsc2VwPSIvIiksIHdpZHRoPTIwLCBoZWlnaHQ9OCwgdW5pdHM9J2NtJywgZGV2aWNlPSd0aWZmJywgZHBpPTcwMCkKYGBg